package com.sunny.crawler.processor;

import com.sohu.idcenter.IdWorker;
import com.sunny.crawler.pipeline.FinancePipeline;
import com.sunny.entity.ApiMetaNameEntity;
import com.sunny.entity.ApiMetaParamEntity;
import com.sunny.entity.ApiMetaResultEntity;
import lombok.extern.java.Log;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;
import java.util.stream.Collectors;

import static com.sunny.config.BeansConfig.crawlerRunningFlag;

@Component
@Log
public class FinanceProcessor implements BaseProcessor {

    public static final String baseUrl = "https://tushare.pro";
    public static final String startUrl = "https://waditu.com/document/2";

    private IdWorker idWorker = new IdWorker();
    @Autowired
    private FinancePipeline financePipeline;

    public void doProcess() {
        //访问入口url地址
        Spider.create(new FinanceProcessor())
                .addUrl(startUrl)
                .setScheduler(new QueueScheduler()
                        .setDuplicateRemover(new BloomFilterDuplicateRemover(10000000)))
                .addPipeline(financePipeline)
                .thread(5)
                .run();
        //爬虫结束
        crawlerRunningFlag.set(false);
        log.info("元数据已经抓取结束");
    }

    @Override
    public void process(Page page) {

        //判断页面是否是api介绍页面
        Request request = page.getRequest();
        //取接口名称
        String name = (String) request.getExtra("name");
        //如果接口名称不为空说明是api介绍页面
        if (StringUtils.isBlank(name)){
            //获取页面数据
            Selectable menu = page.getHtml().$("div#jstree > ul > li");
            //如果页面是菜单页
            if (menu.match()) {
                menu.nodes().forEach(e -> {
                    //只处理直接有菜单项的父节点
                    Selectable items = e.css("a");
                    //取所有子菜单项
                    List<Selectable> aList = items.nodes();
                    aList.forEach(a -> {
                        //获得详情页面链接
                        Request r = new Request();
                        r.setUrl(a.links().get());
                        r.putExtra("category", e.css("li>a", "text").get());
                        r.putExtra("name", a.css("a", "text").get());
                        page.addTargetRequest(r);
                        //跳过pipeline处理
                        page.setSkip(true);
                    });
                });
            }
        } else {
            parseMetaInfo(page);
        }


    }

    @Override
    public Site getSite() {
        return Site.me();
    }

    /**
     * 解析api页面的接口定义信息
     * @param page
     */
    private void parseMetaInfo(Page page) {
        Html html = page.getHtml();
        //解析名称信息
        ApiMetaNameEntity nameEntity = new ApiMetaNameEntity();
        String name = html.css("h2", "text").get();
        String text = html.css("div.content p:nth-of-type(1)").get();
        //判断是否是api介绍页面
        if (StringUtils.isBlank(text) || (!text.contains("接口："))) {
            //不是接口介绍页面跳过
            page.getResultItems().setSkip(true);
            return;
        }
        String code = text.substring(6, text.indexOf("<br>"));
        text = text.substring(text.indexOf("描述"), text.indexOf("</p>"));
        String desc = text.substring(3);
        nameEntity.setApiName(name);
        nameEntity.setApiCode(code);
        nameEntity.setApiDesc(desc);
        nameEntity.setApiCategory((String) page.getRequest().getExtra("category"));
        //此处需要生成ID
        long apid = idWorker.getId();
         nameEntity.setApiId(apid);
        page.getResultItems().put("name", nameEntity);
        //解析输入参数
        Selectable tParam = html.css("table:nth-of-type(1) > tbody > tr");
        if (tParam.match()) {
            List<ApiMetaParamEntity> paramList = tParam.nodes()
                    .stream()
                    .map(tr -> {
                        List<Selectable> tds = tr.css("td").nodes();
                        ApiMetaParamEntity paramEntity = new ApiMetaParamEntity();
                        paramEntity.setParamName(tds.get(0).css("td","text").get());
                        paramEntity.setParamType(tds.get(1).css("td","text").get());
                        paramEntity.setRequired(tds.get(2).css("td","text").get());
                        paramEntity.setParamDesc(tds.get(3).css("td","text").get());
                        paramEntity.setApiId(apid);
                        return paramEntity;
                    }).collect(Collectors.toList());
            page.getResultItems().put("param", paramList);
        }
        //解析返回结果
        Selectable tResult = null;
        List<Selectable> tableList = html.css("table").nodes();
        for(int i=1;i<tableList.size();i++) {
            Selectable table = tableList.get(i);
            String col1 = table.css("thead>tr>th", "text").nodes().get(0).get();
            //判断第二、三个表格中的第一个字段是否是“名称”，
            // 如果是就取这个表格中的内容
            if ("名称".equals(col1)) {
                tResult = table;
                break;
            }
        }
        Selectable thead = tResult.css("thead>tr>th");
        //取第三列的标题
        String col3Text = thead.nodes().get(2).css("th", "text").get();

        if (tResult.match()) {

            //取内容列表
            List<ApiMetaResultEntity> resultEntityList = tResult.css("tbody>tr")
                    .nodes()
                    .stream()
                    .map(tr -> {
                        List<Selectable> tds = tr.css("td").nodes();
                        ApiMetaResultEntity resultEntity = new ApiMetaResultEntity();
                        resultEntity.setResName(tds.get(0).css("td","text").get());
                        resultEntity.setResType(tds.get(1).css("td","text").get());
                        resultEntity.setApiId(apid);
                        if (col3Text.contains("默认")) {
                            resultEntity.setDefaultShow(tds.get(2).css("td","text").get());
                            resultEntity.setResDesc(tds.get(3).css("td","text").get());
                        } else {
                            resultEntity.setResDesc(tds.get(2).css("td","text").get());
                        }
                        return resultEntity;
                    }).collect(Collectors.toList());
            page.getResultItems().put("result", resultEntityList);

        }
    }


}